package com.chance.cc.crawler.development.scripts.kaoyan;

import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.record.builder.CrawlerRequestRecordBuilder;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Selectable;
import com.chance.cc.crawler.core.tags.CategoryTag;
import com.chance.cc.crawler.core.tags.crawler.CrawlerResultTags;
import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.text.ParseException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.*;
import static com.chance.cc.crawler.development.scripts.allfeild.AICCommonField.*;

/**
 * @Author Zhao.Hhuan
 * @Date Create in 2020/12/9 16:07
 * @Description 考研论坛
 **/
public class KaoyanCrawlerScript extends CrawlerCommonScript {

    private Logger log = LoggerFactory.getLogger(KaoyanCrawlerScript.class);

    private static final String kaoyanEntranceUrl = "http://bbs.kaoyan.com/forum.php\\?mod=forumdisplay\\S*";
    private static final String articleUrl = "http://bbs.kaoyan.com/forum.php\\?mod=viewthread\\S*lastpost";
    private static final String commentUrl = "http://bbs.kaoyan.com/forum.php\\?mod=viewthread\\S*page=\\d+";

    /**
     * 脚本domain定义
     *
     * @return
     */
    @Override
    public String domain() {
        return "kaoyan";
    }

    /**
     * 进入脚本的正则列表
     */
    @Override
    public void initUrlRegulars() {
        addUrlRegular(kaoyanEntranceUrl);
        addUrlRegular(articleUrl);
        addUrlRegular(commentUrl);
    }

    /**
     * 是否执行脚本 ： 输入数据检查，合格的才进入脚本
     *
     * @param crawlerRequestRecord
     * @return
     */
    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRequestRecord) {
        return true;
    }


    /**
     * 解析链接方法
     *
     * @param crawlerRequestRecord
     * @param httpPage
     * @return
     */
    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        List<CrawlerRequestRecord> parsedLinks = new ArrayList<CrawlerRequestRecord>();
        CategoryTag categoryTag = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag();

        boolean downloadSuccess = httpPage.isDownloadSuccess();
        if (downloadSuccess) {
            String requestUrl = httpPage.getRequest().getUrl();
            int statusCode = httpPage.getStatusCode();

            if (statusCode == 200) {
                if (requestUrl.matches(kaoyanEntranceUrl)) {
                    //下一页地址的解析
                    String nextPageUrl = httpPage.getHtml().xpath("//a[@class=\"nxt\"]/@href").get();
                    if (StringUtils.isNotEmpty(nextPageUrl)) {
                        nextPageUrl = StringEscapeUtils.unescapeHtml4(nextPageUrl);
                        CrawlerRequestRecord turnPageRecord = CrawlerRequestRecord.builder()
                                .turnPageRequest(crawlerRequestRecord)
                                .recordKey(nextPageUrl)
                                .httpUrl(nextPageUrl)
                                .needWashed(false)
                                .copyBizTags()
                                .releaseTime(System.currentTimeMillis())
                                .build();

                        parsedLinks.add(turnPageRecord);
                    }

                    //文章列表解析
                    List<Selectable> nodes = httpPage.getHtml().xpath("//tbody[contains(@id,'normalthread')]").nodes();
                    for (Selectable node : nodes) {
                        String order = categoryTag.getKVTagStrVal("order");
                        String itemUrl = node.xpath(".//a[@class=\"s xst\"]/@href").get();
                        if (StringUtils.isEmpty(itemUrl)) {
                            continue;
                        }

                        String releaseTime = "";
                        if("reply".equals(order)){
                            //最新回复时间
                            releaseTime = node.xpath(".//td[@class=\"by\"]/em/a/text()").get();
                            itemUrl = StringEscapeUtils.unescapeHtml4(itemUrl);
                        }else{
                            //最新发帖时间
                            itemUrl = StringEscapeUtils.unescapeHtml4(itemUrl).replace("author","lastpost").replace("dateline","lastpost");
                            releaseTime = node.xpath(".//td[@class=\"by\"]/em/span").get();
                        }
                        if (StringUtils.isEmpty(releaseTime)) {
                            continue;
                        }

                        try {
                            long releaseTimeToLong = DateUtils.parseDate(releaseTime, "yyyy-MM-dd HH:mm").getTime();
                            CrawlerRequestRecordBuilder crawlerRequestRecordBuilder = CrawlerRequestRecord.builder()
                                    .itemPageRequest(crawlerRequestRecord)
                                    .recordKey(itemUrl)
                                    .httpUrl(itemUrl)
                                    .resultLabelTag(article)
                                    .resultLabelTag(interaction)
                                    .copyBizTags()
                                    .releaseTime(releaseTimeToLong);

                            CrawlerRequestRecord itemRecord = crawlerRequestRecordBuilder.build();

                            parsedLinks.add(itemRecord);
                        } catch (ParseException e) {
                            log.warn("文章(“" + itemUrl + "”)的时间格式错误！时间是：" + releaseTime);
                        }
                    }
                }else if(requestUrl.matches(articleUrl)){
                    //判断是否有回复是否多页（获取到最后一页的地址）
                    String lastCommentUrl = httpPage.getHtml().xpath("//div[@class=\"pgs mtm mbm cl\"]/div[@class=\"pg\"]/label/preceding-sibling::a[1]/@href").get();
                    if(StringUtils.isNotEmpty(lastCommentUrl)){
                        lastCommentUrl = StringEscapeUtils.unescapeHtml4(lastCommentUrl);
                        CrawlerRequestRecord itemRecord = CrawlerRequestRecord.builder()
                                .itemPageRequest(crawlerRequestRecord)
                                .recordKey(lastCommentUrl)
                                .httpUrl(lastCommentUrl)
                                .resultLabelTag(comment)
                                .copyBizTags()
                                .releaseTime(crawlerRequestRecord.getReleaseTime())
                                .build();

                        parsedLinks.add(itemRecord);
                    }else{
                        crawlerRequestRecord.tagsCreator().resultTags().addResultDataType(comment);
                        log.info("文章(“"+requestUrl+"”)评论只有一页！");
                    }
                }else if(requestUrl.matches(commentUrl)){
                    //下一页评论
                    String[] split = requestUrl.split("page=");
                    int page = Integer.parseInt(split[1]);
                    if(page > 1){
                        String nextCommentUrl = split[0] + "page=" + (page - 1);

                        CrawlerRequestRecordBuilder crawlerRequestRecordBuilder = CrawlerRequestRecord.builder()
                                .itemPageRequest(crawlerRequestRecord)
                                .recordKey(nextCommentUrl)
                                .httpUrl(nextCommentUrl)
                                .resultLabelTag(comment)
                                .copyBizTags()
                                .releaseTime(crawlerRequestRecord.getReleaseTime());

                        if(page - 1 == 1){
                            crawlerRequestRecordBuilder.needParsed(false);
                        }
                        CrawlerRequestRecord itemRecord = crawlerRequestRecordBuilder.build();

                        parsedLinks.add(itemRecord);
                    }
                }
            } else {
                crawlerRequestRecord.setNeedWashPage(false);
                log.info("页面(“ " + requestUrl + "”)下载错误！状态码：" + statusCode);
            }
        }
        return parsedLinks;
    }

    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerDataList = new ArrayList<>();
        CrawlerResultTags crawlerResultTags = crawlerRecord.tagsCreator().resultTags();

        int statusCode = page.getStatusCode();
        String requestUrl = page.getRequest().getUrl();
        if (statusCode == 200) {
            List<Selectable> nodes = page.getHtml().xpath("//div[@id=\"postlist\"]/div[contains(@id,'post_')]").nodes();
            if(nodes.size() > 0){
                if (crawlerResultTags.hasDataType(article)) {
                    crawlerDataList.add(washArticle(crawlerRecord, page));
                }

                if (crawlerResultTags.hasDataType(interaction)) {
                    crawlerDataList.add(washInteraction(crawlerRecord, page));
                }

                if (crawlerResultTags.hasDataType(comment)) {
                    int allPage = 1;
                    if(requestUrl.matches(commentUrl)){
                        allPage = 0;
                    }
                    if(nodes.size() > allPage){
                        for (int i = nodes.size() - 1; i >= allPage; i--) {
                            crawlerDataList.add(washComment(crawlerRecord,page,nodes.get(i)));
                        }
                    }else{
                        log.info("这个评论页(“"+requestUrl+"”)没有评论信息!");
                    }
                }
            }else{
                log.error("指定的主题不存在或已被删除或正在审核！(“"+requestUrl+"”)");
            }
        }


        return crawlerDataList;
    }

    public CrawlerData washArticle(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        CategoryTag categoryTag = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag();

        String itemUrl = httpPage.getRequest().getUrl();

        String title = httpPage.getHtml().xpath("//span[@id=\"thread_subject\"]/text()").get();

        List<Selectable> nodes = httpPage.getHtml().xpath("//div[@id=\"postlist\"]/div[contains(@id,'post_')]").nodes();

        Selectable selectable = nodes.get(0);//楼主的评论
        String author = selectable.xpath(".//a[@class=\"xw1\"]/text()").get();
        String releaseTime = selectable.xpath(".//em[@id]").get();
        if(StringUtils.isNotEmpty(releaseTime) && releaseTime.contains("发表于")){
            releaseTime = getString("\\d{4}-\\d+-\\d+ \\d+:\\d+",releaseTime);
        }

        List<String> articleTextList = nodes.get(0).xpath(".//td[contains(@id,'postmessage')]/text()").all();
        StringBuffer conents = new StringBuffer();
        int size = articleTextList.size();
        for (int i = 0; i < size; i++) {
            if (StringUtils.isNotEmpty(articleTextList.get(i))) {
                conents.append(StringEscapeUtils.unescapeHtml4(articleTextList.get(i))).append(" ");
            }
        }


        String articleKey = getString("tid=\\d+",itemUrl);
        assert articleKey != null;
        articleKey = articleKey.substring(articleKey.lastIndexOf("=") + 1);

        CrawlerData crawlerData = null;
        try {
            assert releaseTime != null;
            crawlerData = CrawlerData.builder()
                    .data(crawlerRequestRecord, httpPage)
                    .dataId(StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), article.enumVal(), articleKey))
                    .resultLabelTag(valueOf("article"))
                    .url(itemUrl)
                    .releaseTime(DateUtils.parseDate(releaseTime, "yyyy-MM-dd HH:mm").getTime())
                    .addContentKV(Field_Content, conents.toString().trim())
                    .addContentKV(Field_Title, StringEscapeUtils.unescapeHtml4(title).trim())
                    .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
                    .requestLabelTag(CrawlerEnum.CrawlerRequestType.filter)
                    .addContentKV(Field_Author, author)
                    .build();

            if("reply".equals(categoryTag.getKVTagStrVal("order"))){
                crawlerData.setFilter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange);
            }

        } catch (ParseException e) {
            log.error("时间格式错误，文章(“" + itemUrl + "”)时间是：" + releaseTime);
        }
        return crawlerData;
    }

    public CrawlerData washInteraction(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        String itemUrl = httpPage.getRequest().getUrl();

        List<String> all = httpPage.getHtml().xpath("//div[@class=\"hm ptn\"]/span/text()").all();
        String views = "0";
        String comments = "0";
        for (int i = 0; i < all.size() - 1; i++) {
            if("查看:".equals(all.get(i))){
                views = all.get(i + 1);
                continue;
            }
            if("回复:".equals(all.get(i))){
                comments = all.get(i + 1);
                continue;
            }
        }
        String articleKey = getString("tid=\\d+",itemUrl);
        assert articleKey != null;
        articleKey = articleKey.substring(articleKey.lastIndexOf("=") + 1);

        CrawlerData crawlerData = CrawlerData.builder()
                .data(crawlerRequestRecord, httpPage)
                .parentId(StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), articleKey))
                .dataId(StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), interaction.enumVal(), articleKey))
                .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("interaction"))
                .releaseTime(crawlerRequestRecord.getReleaseTime())
                .url(itemUrl)
                .addContentKV(Field_I_Views,views)
                .addContentKV(Field_I_Comments,comments)
                .build();

        return crawlerData;
    }

    public CrawlerData washComment(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage, Selectable data) {
        CategoryTag categoryTag = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag();

        String itemUrl = httpPage.getRequest().getUrl();

        String author = data.xpath(".//a[@class=\"xw1\"]/text()").get();
        String releaseTime = data.xpath(".//em[@id]").get();
        if(StringUtils.isNotEmpty(releaseTime) && releaseTime.contains("发表于")){
            releaseTime = getString("\\d{4}-\\d+-\\d+ \\d+:\\d+",releaseTime);
        }

        List<String> articleTextList = data.xpath(".//td[contains(@id,'postmessage')]/text()").all();
        StringBuffer conents = new StringBuffer();
        int size = articleTextList.size();
        for (int i = 0; i < size; i++) {
            if (StringUtils.isNotEmpty(articleTextList.get(i))) {
                conents.append(StringEscapeUtils.unescapeHtml4(articleTextList.get(i))).append(" ");
            }
        }


        String articleKey = getString("tid=\\d+",itemUrl);
        assert articleKey != null;
        articleKey = articleKey.substring(articleKey.lastIndexOf("=") + 1);

        String commentId = data.xpath("./@id").get();
        if(StringUtils.isNotEmpty(commentId)){
            commentId = commentId.substring(commentId.indexOf("_") + 1);
        }

        CrawlerData crawlerData = null;
        try {
            assert releaseTime != null;
            crawlerData = CrawlerData.builder()
                    .data(crawlerRequestRecord, httpPage)
                    .parentId(StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), articleKey))
                    .dataId(StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), comment.enumVal(), commentId))
                    .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("comment"))
                    .releaseTime(DateUtils.parseDate(releaseTime, "yyyy-MM-dd HH:mm").getTime())
                    .url(itemUrl)
                    .requestLabelTag(CrawlerEnum.CrawlerRequestType.filter)
                    .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
                    .addContentKV(Field_Author, author)
                    .addContentKV(Field_Content, conents.toString().trim())
                    .build();

            if("reply".equals(categoryTag.getKVTagStrVal("order"))){
                crawlerData.setFilter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange);
            }
        } catch (ParseException e) {
            log.error("时间格式错误!正确时间是：" + releaseTime);
        }

        return crawlerData;
    }


    @Override
    public void afterExecute(CrawlerRecordContext crawlerRecordContext) {

    }

    /**
     * 获取指定格式的字符串
     *
     * @param regx
     * @param input
     * @return
     */
    private static String getString(String regx, String input) {
        Pattern compile = Pattern.compile(regx);
        Matcher matcher = compile.matcher(input);
        List<String> urlList = new ArrayList<>();
        while (matcher.find()) {
            return matcher.group(0);
        }
        return null;
    }

}
